package com.zhenjiashi.sideline.cralwer.shixian;

import com.zhenjiashi.sideline.cralwer.DataPipeLine;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;

@Component
public class ShixianScheduler {

    @Autowired
    DataPipeLine springDataPipeLine;

    private static final String path = "https://shixian.com/jobs/part-time";

    @Scheduled(cron = "0 0/1 * * * ?")
    public void execute() {
        Spider spider = Spider.create(new ShixianProcessor()).addUrl(path);
        spider.setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)));
        spider.thread(10);
        spider.addPipeline(springDataPipeLine);

        spider.run();
    }

}
